//===-- mart/tools/Mart-Selection.cpp - Main Mutant Selection Module. -----===//
//
//                MART Multi-Language LLVM Mutation Framework
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//
///
/// \file
/// Main source file that implement Mutants Selections
///
//===----------------------------------------------------------------------===//

#include <ctime>
#include <fstream>
#include <sstream>
#include <string>
#include <sys/stat.h>  //mkdir, stat
#include <sys/types.h> //mkdir, stat
//#include <libgen.h> //basename
//#include <unistd.h>  // fork, execl
//#include <sys/wait.h> //wait

#include "../lib/mutantsSelection/MutantSelection.h"
#include "ReadWriteIRObj.h"

#include "llvm/Support/CommandLine.h" //llvm::cl

#include "llvm/Support/FileSystem.h" //for llvm::sys::fs::directory_iterator
#include "llvm/Support/Path.h"       //for llvm::sys::path::filename

using namespace mart;
using namespace mart::selection;

#define TOOLNAME "Mart-Selection"
#include "tools_commondefs.h"

static const std::string selectionFolder("Selection.out");
static const std::string generalInfo("info");
static const std::string readmefile("README.md");
static const std::string defaultFeaturesFilename("mutants-features.csv");
static const std::string defaultStmtFeaturesFilename("stmts-features.csv");
static const std::string
    defaultTrainedModel("trained-models/default-trained.model");
static const std::string
    issta2017TrainedModel("trained-models/issta2017.model");
static const std::string
    mutantTypeOnlyTrainedModel("trained-models/mutant-type-only.model");
static const std::string
    defectPredictionTrainedModel("trained-models/defect-prediction.model");
static const std::string
    equivalentMutantsTrainedModel("trained-models/equivalent-mutants.model");
static const std::string
    subsumingMutantsTrainedModel("trained-models/subsuming-mutants.model");
static const std::string
    hardtokillMutantsTrainedModel("trained-models/hardtokill-mutants.model");
static std::stringstream loginfo;
static std::string outFile;

template <typename T>
void mutantListAsJsON(std::vector<std::vector<T>> const &lists,
                      std::string jsonName) {
  std::ofstream xxx(jsonName);
  if (xxx.is_open()) {
    xxx << "{\n";
    for (unsigned repet = 0, repeat_last = lists.size() - 1;
         repet <= repeat_last; ++repet) {
      xxx << "\t\"" << repet << "\": [";
      bool isNotFirst = false;
      for (T data : lists[repet]) {
        if (isNotFirst)
          xxx << ", ";
        else
          isNotFirst = true;
        xxx << data;
      }
      if (repet == repeat_last)
        xxx << "]\n";
      else
        xxx << "],\n";
    }
    xxx << "\n}\n";
    xxx.close();
  } else {
    llvm::errs() << "Unable to create info file:" << jsonName << "\n";
    assert(false);
  }
}

int main(int argc, char **argv) {
// Remove the option we don't want to display in help
#if (LLVM_VERSION_MAJOR <= 3) && (LLVM_VERSION_MINOR < 5)
  llvm::StringMap<llvm::cl::Option *> optMap;
  llvm::cl::getRegisteredOptions(optMap);
#else
  llvm::StringMap<llvm::cl::Option *> &optMap =
      llvm::cl::getRegisteredOptions();
#endif
  for (auto &option : optMap) {
    auto optstr = option.getKey();
    if (!(optstr.startswith("help") || optstr.equals("version")))
      optMap[optstr]->setHiddenFlag(llvm::cl::Hidden);
  }

  llvm::cl::opt<std::string> martOutTopDir(
      llvm::cl::Positional, llvm::cl::Required,
      llvm::cl::desc("<mutation topdir or out topdir>"));
  llvm::cl::opt<std::string> mutantInfoJsonfile(
      "mutant-infos",
      llvm::cl::desc("Specify the mutants info JSON file. (useful if running "
                     "with topdir different that the one generated by Mart "
                     "mutation)"),
      llvm::cl::value_desc("filename"), llvm::cl::init(""));
  llvm::cl::opt<std::string> inputIRfile(
      "preprocessed-bc-file",
      llvm::cl::desc("specify the pathfile of the preprocessed BC file. "
                     "(useful if running with topdir different that the one "
                     "generated by Mart mutation)"),
      llvm::cl::value_desc("BC filename"), llvm::cl::init(""));
  llvm::cl::opt<bool> mut_dep_cache(
      "mutant-dep-cache",
      llvm::cl::desc("Enable caching of mutant dependence computation"));
  llvm::cl::opt<unsigned> numberOfRandomSelections(
      "rand-repeat-num",
      llvm::cl::desc("(optional) Specify the number of repetitions for random "
                     "selections: default is 100 times"),
      llvm::cl::init(100));
  llvm::cl::opt<std::string> smartSelectionTrainedModel(
      "smart-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for smart"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> issta2017SelectionTrainedModel(
      "issta2017-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for issta2017"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> mutantTypeOnlySelectionTrainedModel(
      "mutant-type-only-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for mutant type only"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> defectPredictionSelectionTrainedModel(
      "defect-prediction-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for defect prediction"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> equivalentMutantsDetectionTrainedModel(
      "equivalent-mutants-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for equivalent mutants"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> subsumingMutantsDetectionTrainedModel(
      "subsuming-mutants-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for subsuming mutants"),
      llvm::cl::init(""));
  llvm::cl::opt<std::string> hardtokillMutantsDetectionTrainedModel(
      "hardtokill-mutants-trained-model",
      llvm::cl::desc(
          "(optional) Specify the alternative to use for prediction for hardtokill mutants"),
      llvm::cl::init(""));
  llvm::cl::opt<bool> dumpMutantsFeaturesToCSV(
      "dump-features",
      llvm::cl::desc("(optional) enable dumping features to CSV file"));
  llvm::cl::opt<bool> disable_selection(
      "no-selection",
      llvm::cl::desc("(optional) Disable selection. useful when only want to "
                     "get the mutants features into a CSV file for training"));
  llvm::cl::opt<bool> disable_mlonly_selection(
      "no-mlonly-selection",
      llvm::cl::desc("(optional) disable ml-only selection"));
  llvm::cl::opt<bool> enable_smart_selection(
      "do-smart-selection",
      llvm::cl::desc("(optional) enable smart selection"));
  llvm::cl::opt<bool> enable_MCL_selection(
      "do-mcl-selection",
      llvm::cl::desc("(optional) enable MCL selection"));
  llvm::cl::opt<bool> enable_ISSTA2017_selection(
      "do-issta2017-selection",
      llvm::cl::desc("(optional) enable ISSTA2017 selection"));
  llvm::cl::opt<bool> enable_mutanttypeonly_selection(
      "do-mutanttypeonly-selection",
      llvm::cl::desc("(optional) enable mutant type only selection"));
  llvm::cl::opt<bool> enable_defectprediction_selection(
      "do-defectprediction-selection",
      llvm::cl::desc("(optional) enable defect prediction selection"));
  llvm::cl::opt<bool> enable_equivalentmutant_detection(
      "do-equivalentmutants-detection",
      llvm::cl::desc("(optional) enable detection of equivalent mutants"));
  llvm::cl::opt<bool> enable_subsumingmutant_detection(
      "do-subsumingmutants-detection",
      llvm::cl::desc("(optional) enable detection of subsuming mutants"));
  llvm::cl::opt<bool> enable_hardtokillmutant_detection(
      "do-hardtokillmutants-detection",
      llvm::cl::desc("(optional) enable detection of hard to kill mutants"));
  llvm::cl::opt<bool> enable_random_selection(
      "do-random-selection",
      llvm::cl::desc("(optional) enable random selection"));

  llvm::cl::SetVersionPrinter(printVersion);

  llvm::cl::ParseCommandLineOptions(argc, argv, "Mart Mutant Selection");

  time_t totalRunTime = time(NULL);
  clock_t curClockTime;

  char *mutantDependencyJsonfile = nullptr;

  bool rundg = true;

  // unsigned numberOfRandomSelections = 100;  //How many time do we repead
  // random

  if (mut_dep_cache)
    mutantDependencyJsonfile = "mutantDependencies.cache.json";

  assert(llvm::sys::fs::is_directory(martOutTopDir) &&
         "Error: the topdir given do not exist!");

  if (mutantInfoJsonfile.empty()) { // try to get it from martOutTopDir
    mutantInfoJsonfile = martOutTopDir + "/" + mutantsInfosFileName;
    // make sure it exists
    assert(llvm::sys::fs::is_regular_file(mutantInfoJsonfile) &&
           "The specified topdir do not contain mutantInfofile and none was "
           "specified");
  } else {
    assert(llvm::sys::fs::is_regular_file(mutantInfoJsonfile) &&
           "The specified mutantInfofile do not exist");
  }
  
  // try to get it from martOutTopDir: end with .bc and is neither .WM.bc 
  // nor .preTCE.MetaMu.bc, nor .MetaMu.bc
  if (inputIRfile.empty()) { 
#if (LLVM_VERSION_MAJOR <= 3) && (LLVM_VERSION_MINOR < 5)
    llvm::error_code ec;
    llvm::sys::fs::directory_iterator dit(martOutTopDir, ec);
#elif (LLVM_VERSION_MAJOR <= 3) && (LLVM_VERSION_MINOR < 9)
    std::error_code ec;
    llvm::sys::fs::directory_iterator dit(martOutTopDir, ec);
#else
    std::error_code ec;
    llvm::sys::fs::directory_iterator dit(martOutTopDir, ec,
                                          false /*no symlink*/);
#endif
    llvm::sys::fs::directory_iterator de;
    for (; dit != de; dit.increment(ec)) {
      assert(!ec && "failed to list topdir when looking for input BC");
      llvm::StringRef fpath = dit->path();
      if (fpath.endswith(commonIRSuffix)) {
        if (!fpath.endswith(wmOutIRFileSuffix) &&
            !fpath.endswith(covOutIRFileSuffix) &&
            !fpath.endswith(preTCEMetaIRFileSuffix) &&
            !fpath.endswith(metaMuIRFileSuffix) &&
            !fpath.endswith(optimizedMetaMuIRFileSuffix)) {
          assert(inputIRfile.empty() && "multiple preprocessed IRs in the "
                                        "specified topdir. Please specify one");
          inputIRfile.assign(martOutTopDir + "/" +
                             llvm::sys::path::filename(fpath).str());
        }
      }
    }
    assert(llvm::sys::fs::is_regular_file(inputIRfile) &&
           "The specified topdir does not contain preprocessed BC file and "
           "none was specified");
  } else {
    assert(llvm::sys::fs::is_regular_file(inputIRfile) &&
           "The specified input BC file do not exist");
  }

  llvm::Module *moduleM;
  std::unique_ptr<llvm::Module> _M;

  // Read IR into moduleM
  /// llvm::LLVMContext context;
  if (!ReadWriteIRObj::readIR(inputIRfile, _M))
    return 1;
  moduleM = _M.get();
  // ~

  MutantInfoList mutantInfo;
  mutantInfo.loadFromJsonFile(mutantInfoJsonfile, true /*fix_missing_srclocs*/);

  std::string outDir(martOutTopDir);
  outDir = outDir + "/" + selectionFolder;
  struct stat st;
  if (stat(outDir.c_str(), &st) == -1) // do not exists
  {
    if (mkdir(outDir.c_str(), 0777) != 0)
      assert(false && "Failed to create output directory");
  } else {
    if (!disable_selection)
      llvm::errs() << "Mart-Selection@Warning: Overriding existing...\n";
  }

  std::string mutDepCacheName;
  if (mutantDependencyJsonfile) {
    mutDepCacheName = outDir + "/" + mutantDependencyJsonfile;
    if (stat(mutDepCacheName.c_str(), &st) != -1) // exists
    {
      rundg = false;
    } else {
      rundg = true;
    }
  } else {
    rundg = true;
  }

  llvm::outs() << "Computing mutant dependencies...\n";
  curClockTime = clock();
  MutantSelection selection(*moduleM, mutantInfo, mutDepCacheName, rundg,
                            false /*is flow-sensitive?*/, disable_selection);
  llvm::outs() << "Mart@Progress: dependencies construction took: "
               << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
               << " Seconds.\n";
  loginfo << "Mart@Progress: dependencies construction took: "
          << (float)(clock() - curClockTime) / CLOCKS_PER_SEC << " Seconds.\n";

  if (dumpMutantsFeaturesToCSV) {
    selection.dumpMutantsFeaturesToCSV(outDir + "/" + defaultFeaturesFilename);
    selection.dumpStmtsFeaturesToCSV(outDir + "/" + defaultStmtFeaturesFilename);
  }

  if (disable_selection) {
    if (!dumpMutantsFeaturesToCSV)
      llvm::errs() << "Mart@Warning: Nothing Done because selection and dump "
                      "features disabled\n";
  } else {
    bool doMLOnly = ! disable_mlonly_selection;
    bool doSmart = enable_smart_selection;
    bool doMCLOnly = enable_MCL_selection;
    bool doMutTypeOnly = enable_mutanttypeonly_selection;
    bool doDefectPrediction = enable_defectprediction_selection;
    bool doISSTA2017 = enable_ISSTA2017_selection;
    bool doRandom = enable_random_selection;
    bool doEquivalentMutants = enable_equivalentmutant_detection;
    bool doSubsumingMutants = enable_subsumingmutant_detection;
    bool doHardtokillMutants = enable_hardtokillmutant_detection;

    if (smartSelectionTrainedModel.empty()) {
      smartSelectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + defaultTrainedModel);
    }
    if (doSmart || doMLOnly)
        assert(llvm::sys::fs::is_regular_file(smartSelectionTrainedModel) && "Smart Selection model file is not found");

    if (issta2017SelectionTrainedModel.empty()) {
      issta2017SelectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + issta2017TrainedModel);
    }
    if (doISSTA2017)
        assert(llvm::sys::fs::is_regular_file(issta2017SelectionTrainedModel) && "Issta2017 selection model file is not found");

    if (mutantTypeOnlySelectionTrainedModel.empty()) {
      mutantTypeOnlySelectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + mutantTypeOnlyTrainedModel);
    }
    if (doMutTypeOnly)
        assert(llvm::sys::fs::is_regular_file(mutantTypeOnlySelectionTrainedModel) && "cwmutant type only selection model file is not found");

    if (defectPredictionSelectionTrainedModel.empty()) {
      defectPredictionSelectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + defectPredictionTrainedModel);
    }
    if (doDefectPrediction)
        assert(llvm::sys::fs::is_regular_file(defectPredictionSelectionTrainedModel) && "Defect pred selection model file is not found");

    if (equivalentMutantsDetectionTrainedModel.empty()) {
      equivalentMutantsDetectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + equivalentMutantsTrainedModel);
    }
    if (doEquivalentMutants)
        assert(llvm::sys::fs::is_regular_file(equivalentMutantsDetectionTrainedModel) && "equivalent mutants detection model file is not found");

    if (subsumingMutantsDetectionTrainedModel.empty()) {
      subsumingMutantsDetectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + subsumingMutantsTrainedModel);
    }
    if (doSubsumingMutants)
        assert(llvm::sys::fs::is_regular_file(subsumingMutantsDetectionTrainedModel) && "subsuming mutants detection model file is not found");

    if (hardtokillMutantsDetectionTrainedModel.empty()) {
      hardtokillMutantsDetectionTrainedModel.assign(getUsefulAbsPath(argv[0]) + "/" + hardtokillMutantsTrainedModel);
    }
    if (doHardtokillMutants)
        assert(llvm::sys::fs::is_regular_file(hardtokillMutantsDetectionTrainedModel) && "hardtokill mutants detection model file is not found");

    std::string smartSelectionOutJson = outDir + "/" + "smartSelection.json";
    std::string mutTypeOnlySelectionOutJson = outDir + "/" + "mutTypeOnlySelection.json";
    std::string defectPredictionSelectionOutJson = outDir + "/" + "defectPredictionSelection.json";
    std::string mlOnlySelectionOutJson = outDir + "/" + "mlOnlySelection.json";
    std::string mclOnlySelectionOutJson = outDir + "/" + "mclOnlySelection.json";
    std::string issta2017SelectionOutJson = outDir + "/" + "issta2017Selection.json";
    std::string scoresForSmartSelectionOutJson =
        outDir + "/" + "scoresForSmartSelection.json";
    //std::string randomSDLelectionOutJson =
    //    outDir + "/" + "randomSDLSelection.json";
    std::string spreadRandomSelectionOutJson =
        outDir + "/" + "spreadRandomSelection.json";
    std::string dummyRandomSelectionOutJson =
        outDir + "/" + "dummyRandomSelection.json";
    std::string scoresForEquivalentmutantsDetectionOutJson =
        outDir + "/" + "scoresForEquivalentMutantsDetection.json";
    std::string scoresForSubsumingmutantsDetectionOutJson =
        outDir + "/" + "scoresForSubsumingMutantsDetection.json";
    std::string scoresForHardtokillmutantsDetectionOutJson =
        outDir + "/" + "scoresForHardtokillMutantsDetection.json";

    std::vector<std::vector<MutantIDType>> selectedMutants1, selectedMutants2;
    unsigned long number = 0;

    std::vector<float> cachedPrediction;

    if (doSmart) {
      llvm::outs() << "Doing Smart Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     smartSelectionTrainedModel,
                                     true /*mlOn*/, true /*mclOn*/, false/*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, smartSelectionOutJson);
      llvm::outs() << "Mart@Progress: smart selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: smart selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";
    }


    if (doMLOnly) {
      llvm::outs() << "Doing ML Only Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     smartSelectionTrainedModel, 
                                     true /*mlOn*/, false /*mclOff*/, false/*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, mlOnlySelectionOutJson);
      llvm::outs() << "Mart@Progress: ML Only selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: ML Only selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";

      number = selectedMutants1.back().size();
      assert(number == mutantInfo.getMutantsNumber() &&
             "The number of mutants mismatch. Bug in Selection function!");

      // write ML's scores
      mutantListAsJsON<float>(std::vector<std::vector<float>>({cachedPrediction}),
                               scoresForSmartSelectionOutJson);
    }

    if (doEquivalentMutants) {
      llvm::outs() << "Doing Equivalent mutants detection...\n";
      curClockTime = clock();
      cachedPrediction.clear();

      selectedMutants1.clear();
      selectedMutants1.resize(1);
      selection.smartSelectMutants(selectedMutants1[0], cachedPrediction,
                                   equivalentMutantsDetectionTrainedModel,
                                   true /*mlOn*/, false /*mclOff*/, false/*dp*/);

      // XXX No need to store mutants order
      
      number = selectedMutants1.back().size();
      assert(number == mutantInfo.getMutantsNumber() &&
             "The number of mutants mismatch. Bug in Selection function!");

      // write ML's scores
      mutantListAsJsON<float>(std::vector<std::vector<float>>({cachedPrediction}),
                               scoresForEquivalentmutantsDetectionOutJson);
    }

    if (doSubsumingMutants) {
      llvm::outs() << "Doing Subsuming mutants detection...\n";
      curClockTime = clock();
      cachedPrediction.clear();

      selectedMutants1.clear();
      selectedMutants1.resize(1);
      selection.smartSelectMutants(selectedMutants1[0], cachedPrediction,
                                   subsumingMutantsDetectionTrainedModel,
                                   true /*mlOn*/, false /*mclOff*/, false/*dp*/);

      // XXX No need to store mutants order
      
      number = selectedMutants1.back().size();
      assert(number == mutantInfo.getMutantsNumber() &&
             "The number of mutants mismatch. Bug in Selection function!");

      // write ML's scores
      mutantListAsJsON<float>(std::vector<std::vector<float>>({cachedPrediction}),
                               scoresForSubsumingmutantsDetectionOutJson);
    }

    if (doHardtokillMutants) {
      llvm::outs() << "Doing Hardtokill mutants detection...\n";
      curClockTime = clock();
      cachedPrediction.clear();

      selectedMutants1.clear();
      selectedMutants1.resize(1);
      selection.smartSelectMutants(selectedMutants1[0], cachedPrediction,
                                   hardtokillMutantsDetectionTrainedModel,
                                   true /*mlOn*/, false /*mclOff*/, false/*dp*/);

      // XXX No need to store mutants order
      
      number = selectedMutants1.back().size();
      assert(number == mutantInfo.getMutantsNumber() &&
             "The number of mutants mismatch. Bug in Selection function!");

      // write ML's scores
      mutantListAsJsON<float>(std::vector<std::vector<float>>({cachedPrediction}),
                               scoresForHardtokillmutantsDetectionOutJson);
    }

    if (doMCLOnly) {
      llvm::outs() << "Doing MCL Only Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     smartSelectionTrainedModel, 
                                     false /*mlOff*/, true /*mclOn*/, false/*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, mclOnlySelectionOutJson);
      llvm::outs() << "Mart@Progress: MCL Only selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: MCL Only selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";
    }
    
    if (doISSTA2017) {
      llvm::outs() << "Doing ISSTA2017 Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     issta2017SelectionTrainedModel, 
                                     true /*mlOff*/, false /*mclOn*/, false/*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, issta2017SelectionOutJson);
      llvm::outs() << "Mart@Progress: ISSTA2017 selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: ISSTA2017 selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";
    }

    if (doMutTypeOnly) {
      llvm::outs() << "Doing MutTypeOnly Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     mutantTypeOnlySelectionTrainedModel,
                                     true /*mlOn*/, false /*mclOn*/, false/*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, mutTypeOnlySelectionOutJson);
      llvm::outs() << "Mart@Progress: mutTypeOnly selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: mutTypeOnly selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";
    }

    if (doDefectPrediction) {
      llvm::outs() << "Doing defectPrediction Selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      // std::vector<double> selectedScores;
      // to make experiment faster XXX: Note to take this in consideration when
      // measuring the algorithm time
      cachedPrediction.clear();
      for (unsigned si = 0; si < numberOfRandomSelections; ++si) {
        selection.smartSelectMutants(selectedMutants1[si], cachedPrediction,
                                     defectPredictionSelectionTrainedModel,
                                     true /*mlOn*/, false /*mclOn*/, true /*dp*/);
      }
      mutantListAsJsON<MutantIDType>(selectedMutants1, defectPredictionSelectionOutJson);
      llvm::outs() << "Mart@Progress: defectPrediction selection took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds.\n";
      loginfo << "Mart@Progress: defectPrediction selection took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
              << " Seconds.\n";
    }

    if (doRandom) {
      llvm::outs() << "Doing dummy and spread random selection...\n";
      curClockTime = clock();
      selectedMutants1.clear();
      selectedMutants2.clear();
      selectedMutants1.resize(numberOfRandomSelections);
      selectedMutants2.resize(numberOfRandomSelections);
      for (unsigned si = 0; si < numberOfRandomSelections; ++si)
        selection.randomMutants(selectedMutants1[si], selectedMutants2[si],
                                number);
      mutantListAsJsON<MutantIDType>(selectedMutants1,
                                     spreadRandomSelectionOutJson);
      mutantListAsJsON<MutantIDType>(selectedMutants2,
                                     dummyRandomSelectionOutJson);
      llvm::outs() << "Mart@Progress: dummy and spread random took: "
                   << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                   << " Seconds. (" << numberOfRandomSelections
                   << " repetitions)\n";
      loginfo << "Mart@Progress: dummy and spread random took: "
              << (float)(clock() - curClockTime) / CLOCKS_PER_SEC << " Seconds. ("
              << numberOfRandomSelections << " repetitions)\n";
    }

    /*llvm::outs()
        << "Doing random SDL selection...\n"; // select only SDL mutants
    curClockTime = clock();
    selectedMutants1.clear();
    selectedMutants1.resize(numberOfRandomSelections);
    for (unsigned si = 0; si < numberOfRandomSelections; ++si)
      selection.randomSDLMutants(selectedMutants1[si], number);
    mutantListAsJsON<MutantIDType>(selectedMutants1, randomSDLelectionOutJson);
    llvm::outs() << "Mart@Progress: random SDL took: "
                 << (float)(clock() - curClockTime) / CLOCKS_PER_SEC
                 << " Seconds. (" << numberOfRandomSelections
                 << " repetitions)\n";
    loginfo << "Mart@Progress: random SDL took: "
            << (float)(clock() - curClockTime) / CLOCKS_PER_SEC << " Seconds. ("
            << numberOfRandomSelections << " repetitions)\n";
    */

    std::ofstream xxx(outDir + "/" + generalInfo);
    if (xxx.is_open()) {
      xxx << loginfo.str();
      xxx.close();
    } else {
      llvm::errs() << "Unable to create info file:"
                   << outDir + "/" + generalInfo << "\n\n";
      assert(false);
    }
  }

  std::ofstream xxx(outDir + "/" + readmefile);
  if (xxx.is_open()) {
    int ind = 1;
    xxx << "## Informations obout the output directory.\n";
    xxx << "```\nMutant IDs are integers greater or equal to 1.\n```\n";
    if (!disable_selection) {
      xxx << ind++ << ". `" << generalInfo << "` file: contain general "
          << "information about the mutant selection run.\n";
      xxx << ind++ << ". `<selction technique>Selection.json file: contains " 
          << "a map where the keys are the repetition IDs and the values "
          << "are the ordered list of mutant, based on the selection "
          << "technique, for that corresponding repetition.\n";
    }
    if (dumpMutantsFeaturesToCSV) {
      xxx << ind++ << ". `" << defaultFeaturesFilename << "` file:"
          << "Contains the mutants features of all mutants from the mutants "
          << "info json file. features row 'i' repesent the features of mutant "
          << "with id 'i'.\n";
      xxx << ind++ << ". `" << defaultStmtFeaturesFilename << "` file:"
          << "Contains the statements features of all executable statements.\n";
      xxx << ind++ << ". `scoresForSmartSelection.json`, "
          << "`scoresForEquivalentMutantsDetection.json`, "
          << "`scoresForSubsumingMutantsDetection` and "
          << "`scoresForHardtokillMutantsDetection.json` files: respectively "
          << " contain the prediction value (positive prediction probability) "
          << "For FaRM fo predict fault revealing, equivalent, subsuming and "
          << "hard-to-kill mutants. The contain is a map with a single key, "
          << "whose value is the list of predicted values. Predicted value at "
          << "index '0' is for mutant ID '1', ..., at index 'i' is for "
          << "mutant ID 'i+1'.\n";
    }
    xxx.close();
  } else {
    llvm::errs() << "Unable to create readme file:"
                 << outDir + "/" + readmefile << "\n\n";
    assert(false);
  }

  llvm::outs() << "@Mart-Selection: Selection Done. Output Directory is "
               << "'" << outDir << "'.\n";

  return 0;
}
